/*
 * @(#)IndexManager.java
 *
 * Copyright (c) 2003 DCIVision Ltd
 * All rights reserved.
 *
 * This software is the confidential and proprietary information of DCIVision
 * Ltd ("Confidential Information").  You shall not disclose such Confidential
 * Information and shall use it only in accordance with the terms of the license
 * agreement you entered into with DCIVision Ltd.
 */
package com.dcivision.framework;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.rmi.Naming;
import java.sql.Timestamp;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Enumeration;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.impl.LogFactoryImpl;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ParallelMultiSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Searchable;
import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.FSDirectory;

import com.dcivision.dms.DmsErrorConstant;
import com.dcivision.framework.bean.AbstractBaseObject;

/**
 * @deprecated This class has deprecated.
 */
public class IndexManager implements Runnable {

  public static final String REVISION = "$Revision: 1.55.2.2 $";

  /*basic index*/
  public final static String INDEX_NAME_UNIQUE_ID = "DCI_UNIQUE_ID";
  public final static String INDEX_NAME_FULLTEXT = "DCI_FULLTEXT";
  public final static String INDEX_NAME_ID = "DCI_ID";
  public final static String INDEX_NAME_OBJECT_TYPE = "DCI_OBJ_TYPE";
  public final static String INDEX_NAME_RECORD_STATUS = "DCI_RECORD_STATUS";
  public final static String INDEX_NAME_UPDATE_DATE = "DCI_UPDATE_DATE";
  public final static String INDEX_NAME_UPDATER = "DCI_UPDATER";
  public final static String INDEX_NAME_UPDATER_ID = "DCI_UPDATER_ID";
  public final static String INDEX_NAME_CREATE_DATE = "DCI_CREATE_DATE";
  public final static String INDEX_NAME_CREATOR = "DCI_CREATOR";
  public final static String INDEX_NAME_CREATOR_ID = "DCI_CREATOR_ID";

  /* extra index constant for advance search sorting*/
  public final static String INDEX_NAME_ITEM_SIZE ="DCI_ITEM_SIZE";
  public final static String INDEX_NAME_ITEM_STATUS = "DCI_ITEM_STATUS";
  public final static String INDEX_NAME_OWNER_ID = "DCI_OWNER_ID";
  public final static String INDEX_NAME_DOC_NAME_SORT = "DCI_DOC_NAME_SORT";

  /* for number and date range search */
  public final static String MAX_DOUBLE_VALUE = "zzzzzzzzzzzzzzzzzzzz";
  public final static String NUMBER_PAD_FORMAT = "00000000000000000000";
  public final static String MAX_DATE_VALUE = "99999999999999";
  public final static String MIN_DATE_VALUE = "00000000000000";

  /* postFix add to the record status to prevent missing Active status "A"
     in query string of Standard Analyiser
   */
// public final static String RECORD_STATUS_PREFIX = "STATUS";

  /*
    Full text highlighter keyword tag marker
  */
  public final static String HIGHLIGHT_MARKER = "KEYWORD" ;

  //Lucene commit lock parameter
  public final static String SYSTEM_PROPERTY_COMMIT_LOCK_TIMEOUT = "org.apache.lucene.commitLockTimeout";
  public final static String SYSTEM_PROPERTY_WRITE_LOCK_TIMEOUT = "org.apache.lucene.writeLockTimeout";

  //Lucene default commit/write timeout period in milliseconds (60000 = 1 minute)
  public final static String DEFAULT_COMMIT_LOCK_TIMEOUT="60000";
  public final static String DEFAULT_WRITE_LOCK_TIMEOUT="60000";

  /** the Logger for logging message */
  private static Log log = new LogFactoryImpl().getInstance(IndexManager.class);

  /**
   * A background thread is switching/updating the index used for indexing
   * and/or searching. The thread sleeps an amount of this constant in
   * milliseconds until the next switch is done.<br/>
   * The shorter the delay, the more actual the search results but also the
   * more preformance overhead is produced.<br/>
   * Be aware that the delay does not determine the index switching frequency
   * because after a sleep of the delay, the index is copied and the switched.
   * This required time for this operation does depend on the size of the
   * index. This also means that the bigger the index, the less acutal are
   * the search results.<br/>
   * A value of 60 minutes (60 * 60 * 1000L) should be OK.
   */

  /** one day in mille second*/
  private static final long OPTIMIZATION_TIME_INTERVAL = 86400000;

  /** the specified time to start the index optimization process */
  private static final String OPTIMIZATION_TIME = "00:00:00";

  /** the name of the configuration tag for the index settings */
  private static final String CONFIGURATION_TAG_INDEXER = "indexer";

  /** the user used to access resources for indexing (global read access) */
  private static final String SEARCH_INDEX_USER = "indexer";

  /**
   * the number of indexes used, must be at least 3:
   * <ul>
   *   <li>one for writing/updating</li>
   *   <li>one for read/search</li>
   *   <li>one temporary where the index is copied to</li>
   * </ul>
   * All further indexes will act as extra backups of the index but will
   * also waste more disk space.
   */
  private static final int INDEX_COUNT = 3;

  /** the read lock object used to synchronize read access (search queries) */
  private Object readLock = null;

  /** the write lock object used to synchronize write access (index modification) */
  private Object writeLock = null;

  /** the default analyzer used for the search index */
  private static final Analyzer ANALYZER = new StandardAnalyzer();
  //private static final CJKAnalyzer ANALYZER = new CJKAnalyzer();

  /** the writer used to add documents to the index */
  private IndexWriter indexWriter = null;

  /** the reader used to remove documents from the index */
  private IndexReader indexReader = null;

  /** the index searcher */
  private Searcher searcher = null;

  /** the path where the indexes is stored */
  private String indexPath;

  /** this flag is set to <code>true</code> if the background thread should stop */
  private boolean done = false;

  /** the background thread that updates the indexes for search */
  private Thread backgroundThread;

  /** Interval of operations to run System.gc(), no GC if = 0  */
  private int gcInterval = 0;
  
  /** Keep how many operations after the last GC  */
  private int operationCount = 0;

  public IndexManager(String indexPath) {
    super();
    this.indexPath = indexPath;
    this.indexWriter = null;
    this.indexReader = null;
    this.done = false;
    this.backgroundThread = null;
    this.readLock = new Object();
    this.writeLock = new Object();

    //define lucene commit lock timeout period
    String commitLockTimeout=SystemParameterFactory.getSystemParameter(SystemParameterConstant.DMS_INDEX_COMMIT_LOCK_TIMEOUT);
    if (commitLockTimeout==null) {
      commitLockTimeout = DEFAULT_COMMIT_LOCK_TIMEOUT;
    }
    System.setProperty(SYSTEM_PROPERTY_COMMIT_LOCK_TIMEOUT,commitLockTimeout);

    //define lucene write lock timeout period
    String writeLockTimeout=SystemParameterFactory.getSystemParameter(SystemParameterConstant.DMS_INDEX_WRITE_LOCK_TIMEOUT);
    if(writeLockTimeout==null) {
      writeLockTimeout = DEFAULT_COMMIT_LOCK_TIMEOUT;
    }
    System.setProperty(SYSTEM_PROPERTY_WRITE_LOCK_TIMEOUT,writeLockTimeout);
    
    String gcIntervalString = SystemParameterFactory.getSystemParameter(SystemParameterConstant.DMS_INDEX_GARBAGE_COLLECTION_INTERVAL);
    if(gcIntervalString!=null) {
      gcInterval = Integer.parseInt(gcIntervalString);
    }    

  }

  public IndexManager(String indexPath, String callFrom) {
    this(indexPath);
    try {
      synchronized (this.readLock) {
        try {
          this.searcher.close();
          this.searcher = null;
        } catch (Exception ignore) {
        }
        this.searcher = new IndexSearcher(IndexReader.open(new File(this.indexPath)));


      }
    } catch (Exception e) {
      log.error(e, e);
    }
  }

  public void start() throws Exception {
    log.info("Start the index engine.");
    try {
      boolean isCreate = false;
      File indexDir = new File(this.indexPath);
      if (!indexDir.exists()) {
        indexDir.mkdirs();
      }
      indexDir = new File(this.indexPath + "/segments");
      if (!indexDir.exists()) {
        isCreate = true;
      } else if (IndexReader.isLocked(FSDirectory.getDirectory(this.indexPath, false))) {
        IndexReader.unlock(FSDirectory.getDirectory(this.indexPath, false));
      }

      synchronized (this.writeLock) {
        this.indexWriter = new IndexWriter(this.indexPath, ANALYZER, isCreate);
        this.indexWriter.setUseCompoundFile(true);
        if (!isCreate) {
          this.indexWriter.optimize();
        }
        this.indexWriter.close();
        this.indexWriter = null;
      }

      synchronized (this.readLock) {
        try {
          this.searcher.close();
          this.searcher = null;
        } catch (Exception ignore) {
        }
        this.searcher = new IndexSearcher(IndexReader.open(new File(this.indexPath)));
      }

      this.backgroundThread = new Thread(this, "Index Switcher");
      this.backgroundThread.start();
    } catch (Exception e) {
      log.error(e, e);
    }
  }

  public void stop() throws Exception {
    log.info("Background Thread Stop.");
    this.done = true;
    this.backgroundThread.interrupt();
    this.backgroundThread = null;
    synchronized (this.writeLock) {
      this.indexWriter = new IndexWriter(new File(this.indexPath), ANALYZER, false);
      this.indexWriter.setUseCompoundFile(true);
      this.indexWriter.optimize();
      this.indexWriter.close();
      this.indexWriter = null;
    }
    synchronized (this.readLock) {
      this.searcher.close();
      this.searcher = null;
    }
  }

  public void run() {
    boolean runOnce= false;
    while (!this.done) {
      try {
        if (runOnce==false) {
          // first time to run the thread, and check the remain time to start
          Thread.sleep(this.getRemainOptimizeTime());
          runOnce=true;
        } else {
          Thread.sleep(OPTIMIZATION_TIME_INTERVAL);
          runOnce=true;
        }
        this.optimize();
      } catch (java.lang.InterruptedException e) {
        log.error("Interrupted Thread of Indexing!", e);
      } catch (java.io.IOException e) {
        log.error("Error During Optimization!", e);
      }
    }
  }

  public synchronized void optimize() throws java.io.IOException {
  	try{
      log.info("Start to optimize the index");
      synchronized (this.writeLock) {
        this.indexWriter = new IndexWriter(new File(this.indexPath), ANALYZER, false);
        this.indexWriter.setUseCompoundFile(true);
        this.indexWriter.optimize();
        this.indexWriter.close();
        this.indexWriter = null;
      }
      log.info("Finished optimize the index");
    }catch(java.io.IOException e){
      throw e;
    }finally{
   	  freeSystemResource();
   	}
  }

  public void dispose() {

  }

  public synchronized void insertDocument(Document doc) throws ApplicationException {
    try {
      log.info("Start to insert a index document");
      synchronized (this.writeLock) {
        this.indexWriter = new IndexWriter(new File(this.indexPath), ANALYZER, false);
        this.indexWriter.setUseCompoundFile(true);
        this.indexWriter.addDocument(doc);
        this.indexWriter.close();
        this.indexWriter = null;
      }
      synchronized (this.readLock) {
        try {
          this.searcher.close();
          this.searcher = null;
        } catch (Exception ignore) {
        }
        this.searcher = new IndexSearcher(IndexReader.open(new File(this.indexPath)));
      }
      log.info("Finished insert a index document");
            
    } catch (Exception e) {
      log.error("Cannot insert index.", e);
      throw new ApplicationException(ErrorConstant.INDEX_CANNOT_WRITE_INDEX, e);
    }finally{
      freeSystemResource();
    }    
  }
  
  public synchronized void updateDocument(Document doc) throws ApplicationException {
    try {
      log.info("Start to update a index document: " + doc.get(INDEX_NAME_UNIQUE_ID));
      Hits hitList = null;
      Document oldDoc = null;
      synchronized (this.readLock) {
        QueryParser qParser = new QueryParser(INDEX_NAME_UNIQUE_ID, new org.apache.lucene.analysis.WhitespaceAnalyzer());
        Query query = qParser.parse(INDEX_NAME_UNIQUE_ID + ":\"" + doc.get(INDEX_NAME_UNIQUE_ID) + "\"");
        log.info("query: " + query);
        hitList = this.searcher.search(query);
      }
      log.info("hitList.length(): " + hitList.length());
      if (hitList.length() > 0) {
        oldDoc = hitList.doc(0);
        HashMap hashMap = new HashMap();

        for (Enumeration docFields = doc.fields(); docFields.hasMoreElements(); ) {
          String fieldName = ((Field)docFields.nextElement()).name();
          hashMap.put(fieldName, doc.get(fieldName));
        }

        for (Enumeration oldDocFields = oldDoc.fields(); oldDocFields.hasMoreElements(); ) {
          String fieldName = ((Field)oldDocFields.nextElement()).name();
          if (hashMap.get(fieldName) == null) {
            doc.add(oldDoc.getField(fieldName));
          }
        }

        this.deleteDocument(oldDoc);
        this.insertDocument(doc);
      }
      log.info("Finished update a index document");
    } catch (Exception e) {
      log.error("Cannot update index.", e);
      throw new ApplicationException(ErrorConstant.INDEX_CANNOT_WRITE_INDEX, e);
    }finally{
      freeSystemResource();
    }    
  }

  public synchronized void deleteDocument(Document doc) throws ApplicationException {
    try {
      log.info("Start to delete a index document");
      int n = 0;
      synchronized (this.writeLock) {
        Term term = new Term(INDEX_NAME_UNIQUE_ID, doc.get(INDEX_NAME_UNIQUE_ID));
        this.indexReader = IndexReader.open(new File(this.indexPath));
        n = this.indexReader.deleteDocuments(term);
        this.indexReader.close();
        this.indexReader = null;
      }
      synchronized (this.readLock) {
        try {
          this.searcher.close();
          this.searcher = null;
        } catch (Exception ignore) {
        }
        this.searcher = new IndexSearcher(IndexReader.open(new File(this.indexPath)));
      }
      log.info("Finished delete a index document: " + n);
    } catch (Exception e) {
      log.error("Cannot delete index.", e);
      throw new ApplicationException(ErrorConstant.INDEX_CANNOT_WRITE_INDEX, e);
    }finally{
      freeSystemResource();
    }    
  }

  public Hits queryHits(Query parsedQuery, SessionContainer sessionCon, String sortAttribute, String sortOrder) throws ApplicationException {
    try {
      Hits hitList = null;
      synchronized (this.readLock) {
        //this.searcher.rewrite(parsedQuery);

        /* add by roy for multiple begin
        IndexSearcher [] searchArr = new IndexSearcher[2];
        searchArr[0] = new IndexSearcher(IndexReader.open(new File(this.indexPath)));
        searchArr[1] = new IndexSearcher(IndexReader.open(new File("R:/")));
        ParallelMultiSearcher multiSearcher1= new ParallelMultiSearcher(searchArr);
          //multiSearcher1.rewrite(parsedQuery);

         add by roy for multiple end */

         // add by roy for remote procedure call

        ParallelMultiSearcher multiSearcher1 = null;
        IndexSearcher [] localSearchArr = new IndexSearcher[1];
        localSearchArr[0] = new IndexSearcher(IndexReader.open(new File(this.indexPath)));
        ParallelMultiSearcher localMultiSearcher = new ParallelMultiSearcher(localSearchArr) ;
        ParallelMultiSearcher searcher = localMultiSearcher;
        String jndiStr = SystemParameterFactory.getSystemParameter(SystemParameterConstant.DMS_GROUP_SERVER_ADDRESS);
        if(!Utility.isEmpty(jndiStr )){
          try{
           // ParallelMultiSearcher  searcher1 =
           //        new ParallelMultiSearcher(new Searchable[]{lookupRemote("//localhost/ParaDM_Parallel")});
            String jndiArr[] = TextUtility.splitString(jndiStr,";");
            ParallelMultiSearcher [] searchArr = new ParallelMultiSearcher[jndiArr.length+1];
            for(int i=0;i<jndiArr.length;i++){
              searchArr[i] = new ParallelMultiSearcher(new Searchable[]{lookupRemote("//"+jndiArr[i])});
            }


            searchArr[jndiArr.length] = localMultiSearcher;

            searcher = new ParallelMultiSearcher(searchArr);
          }catch(Exception e){
            log.error("Error in remote search.", e);
            throw new ApplicationException(DmsErrorConstant.REMOTE_SEARCHER_FOUND_ERROR, e);
          }
        }
         // add by roy for remote procedure call



        /* used for sorting of the result list begin */
        Sort sort = new Sort();
        boolean reverse = false; //set as ascending order default
        if((!Utility.isEmpty(sortOrder)) && ("DESC".equals(sortOrder))) {
          reverse=true;
        }

        if((Utility.isEmpty(sortAttribute))||(Utility.isEmpty(sortOrder))) {
          //set the sorting order by score dynamically
          sort.setSort(SortField.FIELD_SCORE);
        } else if("DOCUMENT_NAME".equals(sortAttribute)) {
          sort.setSort(IndexManager.INDEX_NAME_DOC_NAME_SORT,reverse);
        } else if("ITEM_SIZE".equals(sortAttribute)) {
          sort.setSort(IndexManager.INDEX_NAME_ITEM_SIZE,reverse);
        } else if("ITEM_STATUS".equals(sortAttribute)) {
          sort.setSort(IndexManager.INDEX_NAME_ITEM_STATUS,reverse);
        } else if("UPDATER_ID".equals(sortAttribute)) {
          sort.setSort(IndexManager.INDEX_NAME_UPDATER,reverse);
        } else if("UPDATE_DATE".equals(sortAttribute)) {
          sort.setSort(IndexManager.INDEX_NAME_UPDATE_DATE,reverse);
        } else {
          sort.setSort(SortField.FIELD_SCORE);
        }

        /* used for sorting of the result list end */
        //hitList = this.searcher.search(parsedQuery,sort);
        hitList = searcher.search(parsedQuery,sort);

      }
      return(hitList);
    } catch (java.io.IOException e) {
      throw new ApplicationException("Search index error!", e);
    }
  }

 /* public List query(Query parsedQuery, SessionContainer sessionCon) throws ApplicationException {
    try {
      Hits hitList = null;
      synchronized (this.readLock) {
        hitList = this.searcher.search(parsedQuery);
      }
      int len = hitList.length();
      if (len == 0) {
        return new ArrayList();
      }

      Vector result = new Vector(len);
      QueryHighlightExtractor highlighter = new QueryHighlightExtractor(parsedQuery, ANALYZER, "<b>", "</b>");
      int highlightFragmentSizeInBytes = 80;
      int maxNumFragmentsRequired = 4;
      String fragmentSeparator = "...";

      for (int i = 0; i < len; i++) {
        try {
          Document doc = hitList.doc(i);
          String objType = doc.get(INDEX_NAME_OBJECT_TYPE);
          Integer objID = new Integer(doc.get(INDEX_NAME_ID));
          if (sessionCon == null || sessionCon.getPermissionManager().isAccessible(objType, objID)) {
            String fullText = doc.get(INDEX_NAME_FULLTEXT);
            String highlightedText = highlighter.getBestFragments(fullText,
                                    highlightFragmentSizeInBytes,maxNumFragmentsRequired,fragmentSeparator);
            IndexResult idxResult = new IndexResult();
            idxResult.setScore(hitList.score(i));
            idxResult.setIndexDocument(doc);
            idxResult.setFullText(fullText);
            idxResult.setHighlightedText(highlightedText);
            result.add(idxResult);
          }
        } catch (java.io.IOException e) {
          log.error("Problems with index", e);
        }
      }
      return result;
    } catch (java.io.IOException e) {
      throw new ApplicationException("Search index error!", e);
    }
  }
*/

  /**
   * createBasicDocument
   *
   * @param obj
   * @return
   * @throws ApplicationException
   */
  public static Document getBasicDocument(AbstractBaseObject obj, String objType) throws ApplicationException {
    Document doc = new Document();

    // Basic fields to be indexed.
    doc.add(new Field(INDEX_NAME_UNIQUE_ID, TextUtility.noNull(GlobalConstant.OBJECT_TYPE_DOCUMENT + "_" + TextUtility.formatIntegerObj(obj.getID())), Field.Store.YES, Field.Index.UN_TOKENIZED));
    doc.add(new Field(INDEX_NAME_OBJECT_TYPE, TextUtility.noNull(objType), Field.Store.YES, Field.Index.UN_TOKENIZED));
    doc.add(new Field(INDEX_NAME_ID, TextUtility.noNull(TextUtility.formatIntegerObj(obj.getID())), Field.Store.YES, Field.Index.UN_TOKENIZED));
    //doc.add(new Field(INDEX_NAME_UPDATE_DATE, TextUtility.noNull(DateField.dateToString(Utility.stripToMinutes(obj.getUpdateDate())))), Field.Store.YES, Field.Index.UN_TOKENIZED);
    doc.add(new Field(INDEX_NAME_UPDATE_DATE, TextUtility.noNull(TextUtility.formatTimestampToIndex(Utility.stripToMinutes(obj.getUpdateDate()))), Field.Store.YES, Field.Index.UN_TOKENIZED));
    doc.add(new Field(INDEX_NAME_UPDATER, TextUtility.noNull(obj.getUpdaterName().toLowerCase()), Field.Store.YES, Field.Index.UN_TOKENIZED));
    doc.add(new Field(INDEX_NAME_UPDATER_ID, obj.getUpdaterID().toString(), Field.Store.YES, Field.Index.UN_TOKENIZED));
    //doc.add(new Field(INDEX_NAME_CREATE_DATE, TextUtility.noNull(DateField.dateToString(Utility.stripToMinutes(obj.getCreateDate())))), Field.Store.YES, Field.Index.UN_TOKENIZED);
    doc.add(new Field(INDEX_NAME_CREATE_DATE, TextUtility.noNull(TextUtility.formatTimestampToIndex(Utility.stripToMinutes(obj.getCreateDate()))), Field.Store.YES, Field.Index.UN_TOKENIZED));
    doc.add(new Field(INDEX_NAME_CREATOR, TextUtility.noNull(obj.getCreatorName()), Field.Store.YES, Field.Index.UN_TOKENIZED));
    doc.add(new Field(INDEX_NAME_CREATOR_ID, obj.getCreatorID().toString(), Field.Store.YES, Field.Index.UN_TOKENIZED));
    doc.add(new Field(INDEX_NAME_RECORD_STATUS, TextUtility.noNull((obj.getRecordStatus()+obj.getRecordStatus()).toLowerCase()), Field.Store.YES, Field.Index.UN_TOKENIZED));

    return doc;
  }

  // Escape the following characters:
  // \ + - && || ! ( ) { } [ ] ^ " ~ * ? :
  public static String formSearchString(String fieldName, String inStr, boolean isExactMatch) {
    String result = inStr;
    String[] token = null;
    boolean hasQuote = false;
   /* if (inStr.indexOf("\"") >= 0) {
      //isExactMatch = true;
      hasQuote = true;
    }*/


    result = TextUtility.replaceString(result, "\\", "\\\\");
    result = TextUtility.replaceString(result, "+", "\\+");
    result = TextUtility.replaceString(result, "-", "\\-");
    result = TextUtility.replaceString(result, "&&", "\\&\\&");
    result = TextUtility.replaceString(result, "||", "\\|\\|");
    result = TextUtility.replaceString(result, "!", "\\!");
    result = TextUtility.replaceString(result, "(", "\\(");
    result = TextUtility.replaceString(result, ")", "\\)");
    result = TextUtility.replaceString(result, ":", "\\:");
    result = TextUtility.replaceString(result, "^", "\\^");
    result = TextUtility.replaceString(result, "[", "\\[");
    result = TextUtility.replaceString(result, "]", "\\]");
    result = TextUtility.replaceString(result, "{", "\\{");
    result = TextUtility.replaceString(result, "}", "\\}");
    result = TextUtility.replaceString(result, "~", "\\~");
    result = TextUtility.replaceString(result, "*", "\\*");
    result = TextUtility.replaceString(result, "?", "\\?");

    result = TextUtility.replaceString(result,"\"","\\\"");


    token = TextUtility.splitString(result, " ");
    if(!isExactMatch) {
      if (token.length == 1) {
        result = fieldName + ":" + result.trim();
        //result += isExactMatch ? "" : "*";
        result += "*";

      } else {
        String tmpResult = "(";
        for (int i = 0; i < token.length; i++) {
          if (!Utility.isEmpty(token[i])) {
            if (i != 0) {
              tmpResult += "OR ";
            }
            tmpResult += fieldName + ":" + token[i].trim() + (isExactMatch ? " " : "* ");
          }
        }
        result = tmpResult + ")";
      }
    } else {
      result = fieldName + ":" + "\"" +result + "\"";
    }

    log.debug("query string in index Manger:" + result.trim());
    return(result);
  }

  public static String formWildcardSearchString(String inStr) {
    return(inStr + "*");
  }

  public static String formDateRangeSearchString(String fieldName, Timestamp lowerDate, Timestamp upperDate, boolean isInclusive) {
    String result = "";
    String openBracket="";
    String closeBracket="";
    String lowerDateRange ="";
    String upperDateRange ="";

    if(isInclusive) {
      openBracket ="[";
      closeBracket="]";
    } else {
      openBracket = "{";
      closeBracket ="}";
    }

    if(Utility.isEmpty(lowerDate)){
     // lowerDateRange = DateField.MIN_DATE_STRING();
     lowerDateRange = MIN_DATE_VALUE;
    }else{
      //lowerDateRange = DateField.dateToString(lowerDate);
      lowerDateRange = TextUtility.formatTimestampToIndex(lowerDate);
    }
    if(Utility.isEmpty(upperDate)){
      //upperDateRange = DateField.MAX_DATE_STRING();
      upperDateRange = MAX_DATE_VALUE;
    }else{
      //upperDateRange = DateField.dateToString(upperDate);
      upperDateRange = TextUtility.formatTimestampToIndex(upperDate);
    }

    result = fieldName + ":" + openBracket + lowerDateRange + " TO "+ upperDateRange + closeBracket;

    return result;
  }

  public static String formNumericRangeSearchString(String fieldName, Double lowerNumber, Double upperNumber, boolean isInclusive) {
    String result = "";
    String openBracket="";
    String closeBracket="";
    String lowerNumberRange ="";
    String upperNumberRange ="";

    if(isInclusive) {
      openBracket ="[";
      closeBracket="]";
    } else {
      openBracket = "{";
      closeBracket ="}";
    }

    if(Utility.isEmpty(lowerNumber)) {
      lowerNumberRange = numberPad(new Double(0));
    } else {
      lowerNumberRange = numberPad(lowerNumber);
    }
    if(Utility.isEmpty(upperNumber)) {
      upperNumberRange = MAX_DOUBLE_VALUE;
    } else {
      upperNumberRange = numberPad(upperNumber);
    }

    result = fieldName + ":" + openBracket + lowerNumberRange + " TO "+ upperNumberRange + closeBracket;

    return result;
  }

  /*
   * numberPad() - method to format the numeric field in text for lucene range searching,
   *               and comparsion.
   * @number - the numeric value to be format
   * @return - the formatted string
   */
  public static String numberPad(Double number) {
    if (number != null) {
      DecimalFormat formatter = new DecimalFormat(NUMBER_PAD_FORMAT);
      int decialMultiplier = 10000;
      double afterMulitple = number.doubleValue()*decialMultiplier ;
      return formatter.format(afterMulitple);
    } else {
      return("");
    }
  }

  public static String numberPad(Integer number) {
    if (number != null) {
      DecimalFormat formatter = new DecimalFormat(NUMBER_PAD_FORMAT);
      return formatter.format(number.intValue());
    } else {
      return("");
    }
  }

  /*
   * getRemainOptimizeTime() - method the check the time wait for start first optimize
   * process
   * @return long MilliSecond of time remaining
   */
  private long getRemainOptimizeTime() {
    /*the time format is in 24 hr based*/
    SimpleDateFormat timerFormat = new SimpleDateFormat("kk:mm:ss");
    long startTime = System.currentTimeMillis();
    long remainTime =0;
    try{
      java.util.Date optimizeTime = timerFormat.parse(OPTIMIZATION_TIME);
      Calendar calNow = new GregorianCalendar();
      Calendar calBase = new GregorianCalendar();

      calNow.setTime(new java.util.Date(startTime));
      calBase.setTime(optimizeTime);
      Calendar calSpecific = new GregorianCalendar(calNow.get(Calendar.YEAR),calNow.get(Calendar.MONTH),calNow.get(Calendar.DATE),calBase.get(Calendar.HOUR_OF_DAY),calBase.get(Calendar.MINUTE),calBase.get(Calendar.SECOND));
      long timeDiff=(calNow.getTimeInMillis()-calSpecific.getTimeInMillis());

      if (timeDiff<0) {
        //time to
        remainTime=Math.abs(timeDiff);

      } else {
       //time pass..
       calSpecific.add(Calendar.DATE,1);
       timeDiff=(calSpecific.getTimeInMillis()-calNow.getTimeInMillis());

       remainTime=timeDiff;
      }
      return remainTime;
    } catch (Exception e) {
      log.error(e, e);
      remainTime = 1000;
    }

    return remainTime;
  }

  public InputStream getFullText(Integer documentID) {
    InputStream data = null;
    Hits hitList = null;
    Document doc = null;

    try{
      synchronized (this.readLock) {
        QueryParser qParser = new QueryParser(INDEX_NAME_UNIQUE_ID, new org.apache.lucene.analysis.WhitespaceAnalyzer());
        Query query = qParser.parse(INDEX_NAME_UNIQUE_ID + ":\"" + GlobalConstant.OBJECT_TYPE_DOCUMENT+"_" +documentID.toString() + "\"");
        log.info("query: " + query);
        hitList = this.searcher.search(query);
      }

      log.info("hitList.length(): " + hitList.length());

      if (hitList.length() > 0) {
        doc = hitList.doc(0);
        if(doc.get("FULL_TEXT")!=null){  
          byte[] bAry = (doc.get("FULL_TEXT")).getBytes("UTF-8");
          data = new ByteArrayInputStream(bAry); 
        }else{
          data = null;
        }
       
      }
    } catch(Exception e) {
      log.error(e, e);
    }

    return data;
  }

  public static String getHtmlHighLight(String keywords, Query query, String encodeHtml,String openHtmlTag, String closeHtmlTag) {
    String rtnText = encodeHtml;
    List correctPhrase = getCorrectPhraseStrings(keywords,query);
    for(int i=0;i<correctPhrase.size();i++){
    	rtnText = TextUtility.wrapString(rtnText,(String)correctPhrase.get(i),openHtmlTag,closeHtmlTag);
    }
    /*
    rtnText = TextUtility.replaceString(rtnText, "<"+HIGHLIGHT_MARKER+">", openHtmlTag);
    rtnText = TextUtility.replaceString(rtnText, "&lt;"+HIGHLIGHT_MARKER+"&gt;", openHtmlTag);

    rtnText = TextUtility.replaceString(rtnText, "</"+HIGHLIGHT_MARKER+">", closeHtmlTag);
    rtnText = TextUtility.replaceString(rtnText, "&lt;/"+HIGHLIGHT_MARKER+"&gt;", closeHtmlTag);
    */
    return rtnText;
  }
  
  /**
   * Parse the lucene query to get a list of PhraseQuery and TermQuery.
   * This method will parse the subqueries recursively.
   * @param query the lucene query to be parsed
   * @return a list of PhraseQuery and TermQuery
   */
  private static List parsePhraseAndTermQuery(Query query){        
    List phaseQueries = new ArrayList();
    parsePhraseAndTermQuery(query,phaseQueries);
    return phaseQueries;
  }

  /**
   * Parse the lucene query to get a list of PhraseQuery and TermQuery
   * This method will parse the subqueries recursively.   
   * @param query the lucene query to be parsed
   * @param phaseQuerys the temp list that is parsed by the calller
   * @return a list of PhraseQuery and TermQuery
   */  
  private static void parsePhraseAndTermQuery(Query query,List phaseQueries){          
    if(query instanceof BooleanQuery){
      BooleanClause[] queryClauses = ((BooleanQuery)query).getClauses();

      for (int i = 0; i < queryClauses.length; i++)
      {
        Query subquery = queryClauses[i].getQuery();
        parsePhraseAndTermQuery(subquery,phaseQueries);
      }        
    }else if(query instanceof PhraseQuery || query instanceof TermQuery){
      phaseQueries.add(query);      	  	
    } 	    	
  }
  
  /**
   * Get a list of phrase strings for highligher   
   * @param keywords the original keyword phrase submitted by user
   * @param query the lucene query 
   * @return a list of phrase string
   */    
  private static List getCorrectPhraseStrings(String keywords, Query query){
    //get all the phrase from a BooleanQuery
    List queries = IndexManager.parsePhraseAndTermQuery(query);
    
    List phraseStrings = new ArrayList();
    for(int i=0;i<queries.size();i++){
      Query tmpQuery = (Query)queries.get(i);
      if(tmpQuery instanceof TermQuery){
         phraseStrings.add(((TermQuery)tmpQuery).getTerm().text());
      }else{
        PhraseQuery pquery = (PhraseQuery)tmpQuery;
        Term[] terms = pquery.getTerms();
        String noSpaceString="";
        String withSpaceString="";
        for(int j=0;j<terms.length;j++){
       	  withSpaceString += terms[j].text()+" ";
    	    noSpaceString   += terms[j].text();    	  
        }
        withSpaceString.trim();
        if(keywords.indexOf(noSpaceString)>=0){
    	    phraseStrings.add(noSpaceString);
        }else{
          phraseStrings.add(withSpaceString.trim());
        }
      }
    }
    return phraseStrings;
  }  

  private static Searchable lookupRemote(String name) throws Exception {
    return (Searchable) Naming.lookup(name);
  }
  
  private synchronized void freeSystemResource(){    
    if(gcInterval>0){
      //only increment the operationCount if force gc is enabled (gcInterval > 0)
      operationCount++;
      
      //run GC if operationCount % gcInterval = 0
	  if(operationCount % gcInterval == 0){	  
	    Runtime runtime = null;
	    boolean isDebugEnabled = log.isDebugEnabled();
	    
	    //check debug level to prevent calling runtime in production envrionment	    
	  	if(isDebugEnabled){
	      runtime = Runtime.getRuntime();
	  	  log.debug("Before GC:"+runtime.freeMemory());
	  	}
	  	//run GC
		System.gc();
		if(isDebugEnabled){
		  log.debug("After GC:"+runtime.freeMemory());
		}
		//reset operationCount
		operationCount = 0 ;
	  }	  
	}  
  }
  

  // For debug testing purpose only.
  public static void main(String[] argv) throws Exception {
    IndexManager im = new IndexManager("D:/ParaDM/file/Index");
    im.start();
  }
  
  // Escape the following characters:
  // \ + - && || ! ( ) { } [ ] ^ " ~ * ? :
  public static String escapeCharacters(String inStr) {
    String result = inStr;
    String[] token = null;
    boolean hasQuote = false;
   /* if (inStr.indexOf("\"") >= 0) {
      //isExactMatch = true;
      hasQuote = true;
    }*/


    result = TextUtility.replaceString(result, "\\", "\\\\");
    result = TextUtility.replaceString(result, "+", "\\+");
    result = TextUtility.replaceString(result, "-", "\\-");
    result = TextUtility.replaceString(result, "&&", "\\&\\&");
    result = TextUtility.replaceString(result, "||", "\\|\\|");
    result = TextUtility.replaceString(result, "!", "\\!");
    result = TextUtility.replaceString(result, "(", "\\(");
    result = TextUtility.replaceString(result, ")", "\\)");
    result = TextUtility.replaceString(result, ":", "\\:");
    result = TextUtility.replaceString(result, "^", "\\^");
    result = TextUtility.replaceString(result, "[", "\\[");
    result = TextUtility.replaceString(result, "]", "\\]");
    result = TextUtility.replaceString(result, "{", "\\{");
    result = TextUtility.replaceString(result, "}", "\\}");
    result = TextUtility.replaceString(result, "~", "\\~");
    result = TextUtility.replaceString(result, "*", "\\*");
    result = TextUtility.replaceString(result, "?", "\\?");

    result = TextUtility.replaceString(result,"\"","\\\"");
    return result;
  }
  
}